/**
 * 爬取帮助文档脚本
 * 从 https://help.dailiantong.com/channels/3.html 爬取帮助文档内容并保存到JSON文件
 */

const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs').promises;
const path = require('path');
const dotenv = require('dotenv');

dotenv.config();

const BASE_URL = process.env.CRAWL_BASE_URL || 'https://help.dailiantong.com';
const START_URL = process.env.CRAWL_START_URL || 'https://help.dailiantong.com/channels/3.html';
const CRAWL_DELAY = parseInt(process.env.CRAWL_DELAY) || 1000;
const OUTPUT_DIR = path.join(__dirname, '../data');

// 存储已访问的URL，避免重复爬取
const visitedUrls = new Set();
// 存储爬取的数据
const helpDocs = [];

/**
 * 获取网页内容
 */
async function fetchUrl(url) {
  try {
    const response = await axios.get(url, {
      timeout: 10000,
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
      }
    });
    return response.data;
  } catch (error) {
    throw new Error(`获取页面失败 ${url}: ${error.message}`);
  }
}

/**
 * 解析HTML，提取标题和内容
 */
function parseHtml(html, url) {
  const $ = cheerio.load(html);
  const result = {
    url: url,
    title: '',
    content: '',
    links: [],
    category: '',
    summary: ''
  };

  // 提取标题
  result.title = $('title').text().trim() || 
                 $('h1').first().text().trim() || 
                 $('h2').first().text().trim() ||
                 '未命名文档';

  // 提取摘要（meta description 或前200个字符）
  result.summary = $('meta[name="description"]').attr('content') || 
                   $('.summary').text().trim() || 
                   '';

  // 提取分类（从URL或页面中）
  const urlMatch = url.match(/\/channels\/(\d+)/) || url.match(/\/category\/([^\/]+)/);
  if (urlMatch) {
    result.category = urlMatch[1] || urlMatch[2];
  }

  // 提取主要内容
  let contentElement = $('article').first();
  if (contentElement.length === 0) {
    contentElement = $('.content').first();
  }
  if (contentElement.length === 0) {
    contentElement = $('.article-content').first();
  }
  if (contentElement.length === 0) {
    contentElement = $('.main-content').first();
  }
  if (contentElement.length === 0) {
    contentElement = $('body');
  }

  // 移除不需要的元素
  contentElement.find('script, style, nav, footer, header, .nav, .footer, .header').remove();
  
  // 获取HTML内容
  result.content = contentElement.html() || contentElement.text().trim();

  // 如果没有内容，使用body的内容
  if (!result.content || result.content.length < 50) {
    $('body').find('script, style, nav, footer, header').remove();
    result.content = $('body').html() || $('body').text().trim();
  }

  // 清理内容
  result.content = result.content
    .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
    .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
    .replace(/<!--[\s\S]*?-->/g, '')
    .trim();

  // 提取所有链接
  $('a[href]').each((i, elem) => {
    let href = $(elem).attr('href');
    if (!href) return;

    // 处理相对路径
    if (href.startsWith('/')) {
      href = BASE_URL + href;
    } else if (!href.startsWith('http')) {
      try {
        const urlObj = new URL(url);
        href = urlObj.origin + '/' + href.replace(/^\.\.?\//, '');
      } catch (e) {
        return;
      }
    }

    // 只处理同域名的链接
    if (href.includes('help.dailiantong.com')) {
      // 移除锚点
      href = href.split('#')[0];
      // 移除查询参数（可选）
      // href = href.split('?')[0];
      if (!result.links.includes(href) && href !== url) {
        result.links.push(href);
      }
    }
  });

  // 如果摘要为空，从内容中提取
  if (!result.summary && result.content) {
    const textContent = $('<div>').html(result.content).text().trim();
    result.summary = textContent.substring(0, 200).replace(/\s+/g, ' ');
  }

  return result;
}

/**
 * 爬取单个页面
 */
async function crawlPage(url) {
  if (visitedUrls.has(url)) {
    return null;
  }

  console.log(`正在爬取: ${url}`);
  visitedUrls.add(url);

  try {
    const html = await fetchUrl(url);
    const parsed = parseHtml(html, url);
    
    if (parsed.title || parsed.content) {
      helpDocs.push({
        url: parsed.url,
        title: parsed.title || '未命名文档',
        content: parsed.content,
        category: parsed.category || 'default',
        summary: parsed.summary || '',
        links: parsed.links,
        sortOrder: helpDocs.length
      });
      console.log(`✓ 成功爬取: ${parsed.title || url}`);
      
      // 递归爬取链接的子页面
      for (const link of parsed.links) {
        if (link.includes('help.dailiantong.com') && 
            (link.includes('/contents/') || link.includes('/channels/') || link.includes('/article/'))) {
          await new Promise(resolve => setTimeout(resolve, CRAWL_DELAY)); // 延迟避免请求过快
          await crawlPage(link);
        }
      }
    }
    
    return parsed;
  } catch (error) {
    console.error(`✗ 爬取失败 ${url}:`, error.message);
    return null;
  }
}

/**
 * 保存数据到JSON文件
 */
async function saveToFile() {
  if (helpDocs.length === 0) {
    console.log('没有数据需要保存');
    return;
  }

  console.log(`\n开始保存 ${helpDocs.length} 个文档到文件...`);
  
  try {
    // 确保输出目录存在
    await fs.mkdir(OUTPUT_DIR, { recursive: true });
    
    // 准备文档数据
    const documents = helpDocs.map(doc => ({
      title: doc.title,
      content: doc.content,
      url: doc.url,
      category: doc.category,
      summary: doc.summary,
      sortOrder: doc.sortOrder,
      links: doc.links,
      isPublished: true
    }));

    // 保存为JSON文件
    const crawlDate = new Date();
    const data = {
      crawlDate: crawlDate.toISOString(),
      totalCount: documents.length,
      baseUrl: BASE_URL,
      startUrl: START_URL,
      documents: documents
    };

    // 生成带日期的文件名
    const dateStr = crawlDate.toISOString().split('T')[0];
    const outputFile = path.join(OUTPUT_DIR, `help-docs-${dateStr}.json`);
    
    await fs.writeFile(outputFile, JSON.stringify(data, null, 2), 'utf-8');
    console.log(`✅ 成功保存 ${documents.length} 个文档到文件: ${outputFile}`);
    
    // 同时保存一个最新的文件（不带日期）
    const latestFile = path.join(OUTPUT_DIR, 'help-docs-latest.json');
    await fs.writeFile(latestFile, JSON.stringify(data, null, 2), 'utf-8');
    console.log(`✅ 同时保存最新版本到: ${latestFile}`);
    
    return outputFile;
  } catch (error) {
    console.error('❌ 保存到文件失败:', error);
    throw error;
  }
}

/**
 * 主函数
 */
async function main() {
  console.log('开始爬取帮助文档...');
  console.log(`起始URL: ${START_URL}`);
  console.log(`基础URL: ${BASE_URL}`);
  console.log(`延迟: ${CRAWL_DELAY}ms`);
  console.log(`输出目录: ${OUTPUT_DIR}\n`);
  
  try {
    // 开始爬取
    await crawlPage(START_URL);
    
    console.log(`\n爬取完成！共爬取 ${helpDocs.length} 个文档`);
    console.log(`已访问URL数: ${visitedUrls.size}`);
    
    // 保存到文件
    const outputFile = await saveToFile();
    
    // 显示统计信息
    console.log('\n统计信息:');
    console.log(`- 总文档数: ${helpDocs.length}`);
    console.log(`- 已访问URL数: ${visitedUrls.size}`);
    if (outputFile) {
      console.log(`- 输出文件: ${outputFile}`);
    }
    
    // 显示文档列表（前10个）
    console.log('\n文档列表（前10个）:');
    helpDocs.slice(0, 10).forEach((doc, index) => {
      console.log(`${index + 1}. ${doc.title} (${doc.url})`);
    });
    if (helpDocs.length > 10) {
      console.log(`... 还有 ${helpDocs.length - 10} 个文档`);
    }
    
    console.log('\n✅ 全部完成！');
    process.exit(0);
  } catch (error) {
    console.error('爬取过程出错:', error);
    process.exit(1);
  }
}

// 运行主函数
if (require.main === module) {
  main();
}

module.exports = { crawlPage, parseHtml, fetchUrl };

